# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html

import json
from pymongo import MongoClient

import pymysql


class XicidailiPipeline(object):
    def process_item(self, item, spider):
        print('dbsgw:')
        return item

        

# mysql
class XicidailiPipelineMysql(object):
    def __init__(self):
        # connection database
        self.connect = pymysql.connect(host='localhost', user='dbsgw', passwd='123456',
                                       db='dbsgw')  # 后面三个依次是数据库连接名、数据库密码、数据库名称
        # get cursor
        self.cursor = self.connect.cursor()
        print("连接数据库成功")

    def process_item(self, item, spider):
        # sql语句
        insert_sql = """
        insert into dbsgw(name) values (%s)
        """
        # 执行插入数据到数据库操作
        self.cursor.execute(insert_sql, (item['name']))
        # 提交，不进行提交无法保存到数据库
        self.connect.commit()

        return item

    def close_spider(self, spider):
        # 关闭游标和连接
        self.cursor.close()
        self.connect.close()


class MyspoderPipeline(object):
    def __init__(self):
        # if spider.name == 'dbsgw':
        self.file = open('a.json', 'w')

    def process_item(self, item):
        # if spider.name == 'dbsgw':
        item = dict(item)
        # json_data = json.dumps(item,ensure_ascii=False) + ',\n'
        json_data = json.dumps(item) + ',\n'
        print('dbsgw:', item)
        self.file.write(json_data)
        return item

    def __del__(self):
        # if spider.name == 'dbsgw':
        self.file.close()


class spiders(object):
    # 爬虫打开的时候
    def open_spider(self, spider):
        if spider.name == 'dbsgw':
            self.file = open('a.json', 'w')

    # 爬虫运行时
    def process_item(self, item, spider):
        if spider.name == 'dbsgw':
            item = dict(item)
            # json_data = json.dumps(item,ensure_ascii=False) + ',\n'
            json_data = json.dumps(item) + ',\n'
            print('dbsgw:', item)
            self.file.write(json_data)

        return item

    # 爬虫关闭时
    def close_spider(self, spider):
        if spider.name == 'dbsgw':
            self.file.close()


# 保存数据到mongodb
class MongoPipline(object):

    def open_spider(self, spider):
        self.client = MongoClient('127.0.0.1', 27017)
        self.db = self.client['dbsgw']
        self.col = self.db['wang']

    def process_item(self, item, spider):
        data = dict(item)
        self.col.insert(data)

        return item

    def close_spider(self, spider):
        self.client.close()
